package com.skytech.component.crawler.scratch.downloader;

import com.skytech.component.crawler.scratch.chrome.TestChromeService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.AbstractDownloader;
import us.codecraft.webmagic.selector.PlainText;

public class HtmlUnitDownloader extends AbstractDownloader {

    private static final Logger log = LoggerFactory.getLogger(HtmlUnitDownloader.class);

    private int retryNum;
    private int threadNum;
    private TestChromeService testChromeService;

    @Override
    public Page download(Request request, Task task) {
        String url = request.getUrl();//设置连接超时时间 ，这里是10S。如果为0，则无限期等待
    /*    HtmlPage htmlpage = null;
        try {
            htmlpage = WebClientUtil.getWebClient().getPage(url);
        } catch (Exception e) {
            log.error("{}：页面请求错误,Exception{}", url, e);
            Page page = new Page();
            page.setRequest(request);
            return page;
        }*/
       String htmlpage=testChromeService.downloadUrl(url);
        //String htmlpagexml = htmlpage.asXml();
        System.out.println("1234567");
        Page page = new Page();
        page.setRawText(htmlpage);
        page.setUrl(new PlainText(url));
        page.setRequest(request);
        page.setStatusCode(200);
        return page;
    }

    @Override
    public void setThread(int threadNum) {
        this.threadNum = threadNum;
    }
}
